﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;

namespace InformationRetrieval.Tokenizer
{
    /// <summary>
    /// Provides methods for splitting English input text into tokens. Has
    /// option of filtering out stopwords.
    /// </summary>
    public class EnglishTokenizer : ITokenizer
    {
        #region Stopwords
        private HashSet<string> _stopwords = new HashSet<string>
        {
            "a", "about", "above", "after", "again", "against", "all", "am", 
            "an", "and", "any", "are", "aren't", "as", "at", "be", "because", 
            "been", "before", "being", "below", "between", "both", "but", "by", 
            "can't", "cannot", "could", "couldn't", "did", "didn't", "do", 
            "does", "doesn't", "doing", "don't", "down", "during", "each", 
            "few", "for", "from", "further", "had", "hadn't", "has", "hasn't", 
            "have", "haven't", "having", "he", "he'd", "he'll", "he's", "her", 
            "here", "here's", "hers", "herself", "him", "himself", "his", "how", 
            "how's", "i", "i'd", "i'll", "i'm", "i've", "if", "in", "into", "is", 
            "isn't", "it", "it's", "its", "itself", "let's", "me", "more", 
            "most", "mustn't", "my", "myself", "no", "nor", "not", "of", "off", 
            "on", "once", "only", "or", "other", "ought", "our", "ours", 
            "ourselves", "out", "over", "own", "same", "shan't", "she", "she'd", 
            "she'll", "she's", "should", "shouldn't", "so", "some", "such", 
            "than", "that", "that's", "the", "their", "theirs", "them", 
            "themselves", "then", "there", "there's", "these", "they", "they'd", 
            "they'll", "they're", "they've", "this", "those", "through", "to", 
            "too", "under", "until", "up", "very", "was", "wasn't", "we", "we'd", 
            "we'll", "we're", "we've", "were", "weren't", "what", "what's", 
            "when", "when's", "where", "where's", "which", "while", "who", 
            "who's", "whom", "why", "why's", "will", "with", "won't", "would", "wouldn't", 
            "you", "you'd", "you'll", "you're", "you've", "your", "yours", 
            "yourself", "yourselves"
        }; 
        #endregion

        #region Tokenize
        /// <summary>
        /// Takes a TextReader instance and splits its contents into tokens.
        /// Tokens are split on all non-alphanumeric characters. 
        /// If 'filterStopwords' is true, stopwords are filted out.
        /// </summary>
        /// <param name="reader">A TextReader instance; must be opened and ready to read</param>
        /// <param name="filterStopwords">When true, stopwords are filtered out</param>
        /// <returns>A (possibly empty) sequence of tokens and their positions</returns>
        public IEnumerable<PositionedToken> Tokenize(TextReader reader, bool filterStopwords)
        {
            char[] buffer = new char[1];
            StringBuilder currentToken = new StringBuilder();
            int streamPosition = 0;
            int tokenStartPos = 0;

            while (reader.Read(buffer, 0, 1) > 0)
            {
                //Add letter if it is a letter or digit, otherwise return the token
                if (char.IsLetter(buffer[0]) || buffer[0] == '\'')
                {
                    if (currentToken.Length == 0)
                        tokenStartPos = streamPosition;
                    currentToken.Append(buffer[0]);
                }
                else if (currentToken.Length > 0)
                {
                    string r = currentToken.ToString().ToLower();
                    currentToken.Clear();

                    //Only return tokens with more than 2 letters, filter stopwords
                    if (r.Length > 2 && (!filterStopwords || !_stopwords.Contains(r)))
                        yield return new PositionedToken(r, tokenStartPos);
                }

                streamPosition++;
            }
            //Only return tokens with more than 2 letters, filter stopwords
            string rr = currentToken.ToString().ToLower();
            if (rr.Length > 2 && (!filterStopwords || !_stopwords.Contains(rr)))
                yield return new PositionedToken(rr, tokenStartPos);
        }
        #endregion
    }
}
